/**
 * @date 2015年12月28日 下午9:33:51
 * @version V1.0
 */
package Commons;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;

/**
 * @ClassName: data_pre_process
 * @Description: 用于预处理文档，去除非英文单词和低高频词,以及去除句子长度过短或者过长的句子
 * @author ffftzh
 * @date 2015年12月28日 下午9:33:51
 *
 */
public class data_pre_process {
    private HashMap<String, Integer> wordCount = new HashMap<String, Integer>();
    private HashSet<String> stopList = new HashSet<String>();
    private ArrayList<String> tag_list = new ArrayList<String>();
    private int frequency_num = 0;
    private int min_sentence_length = 0;
    private int max_sentence_length = 0;
    private String path = "";
    private String tag_path = "";

    public data_pre_process(String path, String tag_path, int frequency_num, int min_sentence_length, int max_sentence_length) throws IOException {
        this.frequency_num = frequency_num;
        this.path = path;
        this.tag_path = tag_path;
        this.min_sentence_length = min_sentence_length;
        this.max_sentence_length = max_sentence_length;

        BufferedReader reader = IOUtil.getReader("common/stopwords_big");
        String line = null;
        while ((line = reader.readLine()) != null) {
            line = line.trim();
            stopList.add(line.toLowerCase());
        }
        reader.close();
    }

    private void read_tag() throws IOException {
        BufferedReader reader = IOUtil.getReader(this.tag_path);
        String line = null;
        while ((line = reader.readLine()) != null) {
            tag_list.add(line);
        }
        reader.close();

    }

    private void read_data() throws IOException {
        read_tag();
        BufferedReader reader = IOUtil.getReader(this.path);
        String line = null;
        while ((line = reader.readLine()) != null) {
            line = line.replaceAll("[^a-zA-Z]+", " ");
            String[] wordlist = line.split("[^\\w]+");
            for (String item : wordlist) {
                String pitem = item.toLowerCase();
                if (pitem.length() < 3 || this.stopList.contains(pitem)) {
                    continue;
                }
                if (!wordCount.containsKey(pitem)) {
                    wordCount.put(pitem, 0);
                }
                wordCount.put(pitem, wordCount.get(pitem) + 1);
            }
        }
        reader.close();

    }

    private void write_data() throws IOException {
        BufferedReader reader = IOUtil.getReader(path);
        BufferedWriter writer = IOUtil.getWriter(path + ".processed");
        BufferedWriter tag_writer = IOUtil.getWriter(tag_path + ".processed");
        String line = null;
        ArrayList<String> write_line = new ArrayList<String>();
        while ((line = reader.readLine()) != null) {
            write_line.clear();
            String[] wordlist = line.split("[^\\w]+");
            for (String item : wordlist) {
                String pitem = item.toLowerCase();
                if (!wordCount.containsKey(pitem)) {
                    continue;
                }
                if (wordCount.get(pitem) < this.frequency_num) {
                    continue;
                }
                write_line.add(pitem);
            }
            String oneTagString = tag_list.remove(0);
            if (write_line.size() > min_sentence_length && write_line.size() < max_sentence_length) {
                writer.write(utils.join(write_line, " ") + "\n");
                tag_writer.write(oneTagString + "\n");
            }
        }
        if (tag_list.size() > 0) {
            throw new IOException("tag num error！");
        }
        reader.close();
        tag_writer.close();
        writer.close();
    }

    public static void pre_process_data(String path, String tagPath, int frequency, int min_sentence_length, int max_sentence_length) throws IOException {
        data_pre_process instance = new data_pre_process(path, tagPath, frequency, min_sentence_length, max_sentence_length);
        instance.read_data();
        instance.write_data();
    }

    public static void main(String[] args) throws IOException {
        String root = "G:\\experiment\\data\\news_o\\";
        pre_process_data(root + "news.data", root + "news.tag", 5, 5, 20);

    }

}
